'''
Created on Feb 4, 2012

@author: Forever Together
'''
import mechanize, os
import hashlib, sys
import re
from CrawlerLib import Log

class SimpleDownload():
    
    @staticmethod
    def download(url, path, getfilenamefunc):
        opener = mechanize.build_opener(mechanize.HTTPRedirectHandler)
        mechanize.install_opener(opener)
        response = mechanize.urlopen(url)
        filename = getfilenamefunc(url)
        

class Download():
    
    logger = Log.getLogger('Download')
    user_agent = 'Mozilla/5.0 (Windows NT 6.1; rv:11.0) Gecko/20100101 Firefox/11.0'
    
    @staticmethod
    def downloadImage(url, path, prefix, headers={}):
        return Download.download(url, path, prefix, '.jpg', ['text/html'], headers)
    
    @staticmethod
    def download(url, path, prefix='', ext='', except_mime=[], headers={}):
        Download.logger.debug('start download url: {0}'.format(url))
        opener = mechanize.build_opener(mechanize.HTTPRedirectHandler)
        request_header = [("User-Agent", Download.user_agent)]
        for i,v in headers.items(): request_header.append((i, v))
        opener.addheaders = request_header
        mechanize.install_opener(opener)
        response = None
        ret = {'path': '', 'size': 0, 'error': 0, 'msg': ''}
        ido = 0
        max_retry = 2
        download_finished = False
        while ido <= max_retry and (download_finished == False):
            try:
                if (ido>0): Download.logger.debug('retry download url {0}'.format(url))
                response = mechanize.urlopen(url)
                headers = response.info()
                content_type = headers.getheader('Content-Type')
                for imime in except_mime:
                    if imime in content_type:
                        ret['error'] = 1
                        ret['msg'] = 'Do not download webpage'
                        return ret
                content_length = long(headers.getheader('Content-Length'))
                if path[-1] != '/': path += '/'
                file_path = path + Download.getMd5FileName(url) + Download.detectFileExtension(headers, ext)
                Download.logger.debug('download {0} to {1}'.format(url, file_path))
                byte_read = 0
                if not os.path.exists(file_path):
                    if not os.path.isdir(os.path.dirname(file_path)): os.makedirs(os.path.dirname(file_path), 0777)
                    fp = open(file_path, 'wb')
                    try:
                        block_size = 4096
                        while 1:
                            block = response.read(block_size)
                            if block == "" or block == None: break
                            byte_read += len(block)
                            fp.write(block)
                    except:
                        raise Exception(sys.exc_info()[0], sys.exc_info()[1])
                    finally:
                        try: fp.close()
                        except: pass
                    del fp
                    if byte_read != content_length: raise Exception, 'file size downloaded diffirent with server response'
                else:
                    byte_read = os.path.getsize(file_path) if os.path.exists(file_path) else 0
                if (len(prefix)>0):
                    if prefix[-1] == '/' and file_path[0] == '/':
                        file_path = prefix[:-1] + file_path
                ret['path'] = '{0}{1}'.format(prefix, file_path)
                ret['size'] = byte_read
                Download.logger.info('downloaded successfully {0}'.format(url))
                download_finished = True
            except:
                ret['error'] = 1
                ret['msg'] = "{0}:{1}".format(sys.exc_info()[0], sys.exc_info()[1])
                Download.logger.warning(ret['msg'])
            finally:
                try: response.close()
                except: pass
            if response != None: del response
            ido += 1
        if (not download_finished):
            Download.logger.warning("download failed with url {0}".format(url))
        return ret
        
    @staticmethod
    def getMd5Path(stringToHash):
        s = hashlib.md5(str(stringToHash)).hexdigest()
        s = '{0}/{1}/{2}/{3}'.format(s[0], s[1], s[2], s[3])
        return s
        
    @staticmethod
    def getMd5FileName(stringToHash):
        s = hashlib.md5(str(stringToHash)).hexdigest()
        s = '{0}/{1}/{2}/{3}/{4}'.format(s[0], s[1], s[2], s[3], s[4:])
        return s
    
    @staticmethod
    def detectFileExtension(headers, default_extension=''):
        content_type = headers.getheader('Content-Type')
        ext = MIME.getFileExtension(content_type)
        if ext == '' and headers.getheader('Content-Disposition') != None:
            attachment = headers.getheader('Content-Disposition')
            ext = attachment[attachment.rfind('.'):]
        if len(ext)>5: ext = default_extension
        return ext
    
class MIME():
    
    @staticmethod
    def getFileExtension(content_type):
        mime_list = {
            'image/bmp': '.bmp',
            'image/gif': '.gif',
            'image/x-icon': '.ico',
            'image/jpeg': '.jpg',
            'image/pjpeg': '.jpg',
            'image/png': '.png',
            'audio/x-midi': '.mid',
            'audio/midi': '.mid',
            'audio/x-mpeg-3': '.mp3',
            'audio/mpeg': '.mp3',
            'audio/mpeg3': '.mp3',
            'video/mpeg': '.mpg',
            'video/mp4': '.mp4',
            'video/x-flv': '.flv',
            'video/quicktime': '.mov',
            'video/avi': '.avi',
            'video/x-msvideo': '.avi',
            'video/msvideo': '.avi',
            'application/x-compressed': '.zip',
            'application/zip': '.zip',
            'multipart/x-zip': '.zip',
            'application/x-zip-compressed': '.zip',
            'text/html': '.html',
            'text/plain': '.txt',
        }
        m = re.match(r'(.+/.+);|\s', content_type)
        if m: content_type = m.group(1)
        return mime_list.get(content_type, '')
    